# -*- coding: utf-8 -*-
import requests,json,os,urllib,string,sys
import scrapy
from zuqiu.items import ZuqiuItem
from zuqiu.sqlHelper import *

reload(sys)
sys.setdefaultencoding('utf8')

class ZqSpider(scrapy.Spider):
    name = 'zq'
    allowed_domains = ['500.com']
    offset=1
    url='http://liansai.500.com/team/'
    start_urls = [url+str(offset)]
    DOWNLOAD_DELAY=0.1
    def parse(self, response):
        for each in response.xpath("//div[@class='itm_logo']"):
            item = ZuqiuItem()
            try:
                cn=each.xpath("./img/@alt").extract()[0]
                en=each.xpath("./div/div/text()").extract()[0]
                url = each.xpath("./img/@src").extract()[0]
                if en[0:1]==' ':
                    en=en[1:len(en)]
                item["en"]=en
                item["state"]=self.offset
                if self.save_data(en,cn):
                    self.save_img(url,en)

                yield item
            except:
                print str(self.offset)+'is not find'

        if self.offset < 10623:
            self.offset += 1
#        print self.url+str(self.offset)
        yield scrapy.Request(self.url+str(self.offset), callback = self.parse)

    def save_data(self,en,cn):
        if sqlHelper.selectOne("select id from team where en=%s and cn=%s",(en,cn)):
#            print cn+'已采集'
            return False
        sqlHelper.update("insert into team(cn,en,logo) values(%s,%s,%s)",(cn,en,en))
        print cn
        return True
    
    def save_img(self,img_url,file_name,file_path='img'):
        img_url=img_url.replace('\/','/')
        try:
            if not os.path.exists(file_path):
                print '文件夹',file_path,'不存在，重新建立'
                #os.mkdir(file_path)
                os.makedirs(file_path)
            #获得图片后缀
            file_suffix = os.path.splitext(img_url)[1]
            #拼接图片名（包含路径）
            filename = '{}{}{}{}'.format(file_path,os.sep,file_name,file_suffix)
            filename=filename.replace('.com','_com')
            if string.find(filename,'.')==-1:
                filename=filename+'.png'
            if string.find(filename,'?')!=-1:
                filename=filename[0:len(filename)-4]
#       m下载图片，并保存到文件夹中
            urllib.urlretrieve(img_url,filename=filename)
        except IOError as e:
            print '文件操作失败',e
        except Exception as e:
            print '错误 ：',e

